diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index bef0aa6a8f474..0bac8f4b160cb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -351,6 +351,7 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro Request request = new Request(HttpPut.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); + params.putParam("include_type_name", Boolean.toString(putIndexTemplateRequest.isCustomTyped())); if (putIndexTemplateRequest.create()) { params.putParam("create", Boolean.TRUE.toString()); } @@ -395,6 +396,9 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { final RequestConverters.Params params = new RequestConverters.Params(request); params.withLocal(getIndexTemplatesRequest.isLocal()); params.withMasterTimeout(getIndexTemplatesRequest.getMasterNodeTimeout()); + if(getIndexTemplatesRequest.includeTypeNamesInResponse() == false) { + params.putParam("include_type_name", Boolean.toString(getIndexTemplatesRequest.includeTypeNamesInResponse())); + } return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesRequest.java index dc00eb56b8477..848e149cc2deb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesRequest.java @@ -40,6 +40,7 @@ public class GetIndexTemplatesRequest implements Validatable { private TimeValue masterNodeTimeout = TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT; private boolean local = false; + private boolean includeTypeNamesInResponse = true; /** * Create a request to read the content of one or more index templates. If no template names are provided, all templates will be read @@ -96,4 +97,13 @@ public boolean isLocal() { public void setLocal(boolean local) { this.local = local; } + + + public boolean includeTypeNamesInResponse() { + return includeTypeNamesInResponse; + } + + public void includeTypeNamesInResponse(boolean includeTypeNamesInResponse) { + this.includeTypeNamesInResponse = includeTypeNamesInResponse; + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 92d7e94394594..053ce1b8c4c74 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -80,6 +80,8 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -1229,8 +1231,25 @@ public void testIndexPutSettingNonExistent() throws IOException { + "reason=unknown setting [index.no_idea_what_you_are_talking_about] please check that any required plugins are installed, " + "or check the breaking changes documentation for removed settings]")); } + - @SuppressWarnings("unchecked") + @SuppressWarnings("deprecation") + public void testPutCustomTypedTemplateHasWarnings() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest() + .name("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .mapping("custom_doc_type", "host_name", "type=keyword", "description", "type=text"); + + + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync)); + assertThat(exception.getDetailedMessage(), containsString("[types removal]")); + assertThat(exception.status(), equalTo(RestStatus.OK)); + } + + @SuppressWarnings({"unchecked","deprecation"}) public void testPutTemplate() throws Exception { PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest() .name("my-template") @@ -1238,23 +1257,48 @@ public void testPutTemplate() throws Exception { .order(10) .create(randomBoolean()) .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) - .mapping("doc", "host_name", "type=keyword", "description", "type=text") + .simplifiedMapping("host_name", "type=keyword", "description", "type=text") .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + IndicesClient indicesClient = highLevelClient().indices(); AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + indicesClient::putTemplate, indicesClient::putTemplateAsync); assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); - Map templates = getAsMap("/_template/my-template"); + + Map templates = getAsMap("/_template/my-template?include_type_name=false"); assertThat(templates.keySet(), hasSize(1)); assertThat(extractValue("my-template.order", templates), equalTo(10)); assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); - assertThat(extractValue("my-template.mappings.doc.properties.host_name.type", templates), equalTo("keyword")); - assertThat(extractValue("my-template.mappings.doc.properties.description.type", templates), equalTo("text")); + assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); + assertThat(extractValue("my-template.mappings.properties.description.type", templates), equalTo("text")); assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); + + // Test the typed version of the template has the default doc type name + templates = getAsMap("/_template/my-template"); + assertThat(extractValue("my-template.mappings._doc.properties.host_name.type", templates), equalTo("keyword")); + assertThat(extractValue("my-template.mappings._doc.properties.description.type", templates), equalTo("text")); + + // Now test with the getIndexTemplatesResponse untyped mappings + GetIndexTemplatesRequest untypedGet = new GetIndexTemplatesRequest("my-template"); + untypedGet.includeTypeNamesInResponse(false); + GetIndexTemplatesResponse getTemplate1 = execute(untypedGet, indicesClient::getTemplate, indicesClient::getTemplateAsync); + IndexTemplateMetaData typelessTemplate = getTemplate1.getIndexTemplates().get(0); + ImmutableOpenMap mappings = typelessTemplate.getMappings(); + assertNull(mappings.get("_doc")); + assertNotNull(mappings.get("properties")); + + // Now test with the getIndexTemplatesResponse typed mappings + GetIndexTemplatesRequest typedMappingRequest = new GetIndexTemplatesRequest("my-template"); + typedMappingRequest.includeTypeNamesInResponse(true); + getTemplate1 = execute(typedMappingRequest, indicesClient::getTemplate, indicesClient::getTemplateAsync); + IndexTemplateMetaData typedTemplate = getTemplate1.getIndexTemplates().get(0); + mappings = typedTemplate.getMappings(); + assertNotNull(mappings.get("_doc")); + assertNull(mappings.get("properties")); } public void testPutTemplateBadRequests() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 010cf04b73766..a00204ddc0fa9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -817,14 +817,16 @@ public void testPutTemplateRequest() throws Exception { if (ESTestCase.randomBoolean()) { putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); } + Map expectedParams = new HashMap<>(); + expectedParams.put("include_type_name", "false"); if (ESTestCase.randomBoolean()) { putTemplateRequest.mapping("doc-" + ESTestCase.randomInt(), "field-" + ESTestCase.randomInt(), "type=" + ESTestCase.randomFrom("text", "keyword")); + expectedParams.put("include_type_name", "true"); // Custom doc types will set the "include_type_name" = true flag } if (ESTestCase.randomBoolean()) { putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); } - Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { expectedParams.put("create", Boolean.TRUE.toString()); putTemplateRequest.create(true); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 8f9d8a069fd48..2ac02c635dccd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2119,16 +2119,14 @@ public void testPutTemplate() throws Exception { { // tag::put-template-request-mappings-json - request.mapping("_doc", // <1> + request.mappingNoDocType( "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + - "}", // <2> + "}", // <1> XContentType.JSON); // end::put-template-request-mappings-json assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); @@ -2136,14 +2134,16 @@ public void testPutTemplate() throws Exception { { //tag::put-template-request-mappings-map Map jsonMap = new HashMap<>(); - Map message = new HashMap<>(); - message.put("type", "text"); - Map properties = new HashMap<>(); - properties.put("message", message); - Map mapping = new HashMap<>(); - mapping.put("properties", properties); - jsonMap.put("_doc", mapping); - request.mapping("_doc", jsonMap); // <1> + { + Map properties = new HashMap<>(); + { + Map message = new HashMap<>(); + message.put("type", "text"); + properties.put("message", message); + } + jsonMap.put("properties", properties); + } + request.mapping(jsonMap); // <1> //end::put-template-request-mappings-map assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } @@ -2152,28 +2152,24 @@ public void testPutTemplate() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); { - builder.startObject("_doc"); + builder.startObject("properties"); { - builder.startObject("properties"); + builder.startObject("message"); { - builder.startObject("message"); - { - builder.field("type", "text"); - } - builder.endObject(); + builder.field("type", "text"); } builder.endObject(); } builder.endObject(); } builder.endObject(); - request.mapping("_doc", builder); // <1> + request.mappingNoDocType(builder); // <1> //end::put-template-request-mappings-xcontent assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } { //tag::put-template-request-mappings-shortcut - request.mapping("_doc", "message", "type=text"); // <1> + request.simplifiedMapping("message", "type=text"); // <1> //end::put-template-request-mappings-shortcut assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } @@ -2192,7 +2188,7 @@ public void testPutTemplate() throws Exception { // end::put-template-request-version // tag::put-template-whole-source - request.source("{\n" + + request.sourceNoDocTypes("{\n" + " \"index_patterns\": [\n" + " \"log-*\",\n" + " \"pattern-1\"\n" + @@ -2202,11 +2198,9 @@ public void testPutTemplate() throws Exception { " \"number_of_shards\": 1\n" + " },\n" + " \"mappings\": {\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + " },\n" + @@ -2269,13 +2263,11 @@ public void testGetTemplates() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template"); putRequest.patterns(Arrays.asList("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3).put("index.number_of_replicas", 1)); - putRequest.mapping("_doc", + putRequest.mappingNoDocType( "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + "}", XContentType.JSON); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index c8220e9cc0c05..8eb49f492b307 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -278,7 +278,7 @@ public void onFailure(Exception e) { client.security().putUserAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::put-user-execute-async - assertTrue(latch.await(30L, TimeUnit.SECONDS)); + assertTrue(latch.await(30L, TimeUnit.SECONDS)); } } diff --git a/docs/java-rest/high-level/indices/put_template.asciidoc b/docs/java-rest/high-level/indices/put_template.asciidoc index 7618fc8ce7af5..f2a8694b827d8 100644 --- a/docs/java-rest/high-level/indices/put_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_template.asciidoc @@ -39,8 +39,7 @@ template's patterns. -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-mappings-json] -------------------------------------------------- -<1> The type to define -<2> The mapping for this type, provided as a JSON string +<1> The mapping for this type, provided as a JSON string The mapping source can be provided in different ways in addition to the `String` example shown above: diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc index 02c0b7a4cbe82..f8709ffaa1805 100644 --- a/docs/reference/indices/templates.asciidoc +++ b/docs/reference/indices/templates.asciidoc @@ -16,25 +16,23 @@ For example: [source,js] -------------------------------------------------- -PUT _template/template_1 +PUT _template/template_1?include_type_name=false { "index_patterns": ["te*", "bar*"], "settings": { "number_of_shards": 1 }, "mappings": { - "_doc": { - "_source": { - "enabled": false + "_source": { + "enabled": false + }, + "properties": { + "host_name": { + "type": "keyword" }, - "properties": { - "host_name": { - "type": "keyword" - }, - "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z yyyy" - } + "created_at": { + "type": "date", + "format": "EEE MMM dd HH:mm:ss Z yyyy" } } } @@ -50,6 +48,11 @@ Defines a template named `template_1`, with a template pattern of `te*` or `bar* The settings and mappings will be applied to any index name that matches the `te*` or `bar*` pattern. +NOTE: This mapping example uses a "typeless" format new to 7.0. +Previous versions of elasticsearch included document type names in the "mappings"section +but here we have set the `include_type_name=false` parameter in the URL to use the new format. +The old approach of including type names in mappings is still supported but is now deprecated. + It is also possible to include aliases in an index template as follows: [source,js] @@ -149,7 +152,7 @@ orders overriding them. For example: [source,js] -------------------------------------------------- -PUT /_template/template_1 +PUT /_template/template_1?include_type_name=false { "index_patterns" : ["*"], "order" : 0, @@ -157,13 +160,11 @@ PUT /_template/template_1 "number_of_shards" : 1 }, "mappings" : { - "_doc" : { - "_source" : { "enabled" : false } - } + "_source" : { "enabled" : false } } } -PUT /_template/template_2 +PUT /_template/template_2?include_type_name=false { "index_patterns" : ["te*"], "order" : 1, @@ -171,9 +172,7 @@ PUT /_template/template_2 "number_of_shards" : 1 }, "mappings" : { - "_doc" : { - "_source" : { "enabled" : true } - } + "_source" : { "enabled" : true } } } -------------------------------------------------- @@ -189,6 +188,11 @@ order templates, with lower order templates providing the basis. NOTE: Multiple matching templates with the same order value will result in a non-deterministic merging order. +NOTE: This mapping example uses the "typeless" format new to 7.0. +Previous versions of elasticsearch included document type names in the "mappings"section +but here we have set the `include_type_name=false` parameter in the URL to use the new format. +The old approach of including type names in mappings is still supported but is now deprecated. + [float] [[versioning-templates]] === Template Versioning diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 7519cade0f29b..ee48471a1ddaf 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.rest.action.document.RestUpdateAction; import org.elasticsearch.rest.action.search.RestExplainAction; @@ -899,8 +900,11 @@ public void testSnapshotRestore() throws IOException { templateBuilder.endObject().endObject(); Request createTemplateRequest = new Request("PUT", "/_template/test_template"); createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder)); + createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + if (false == isRunningAgainstOldCluster()) { + createTemplateRequest.addParameter("include_type_name", Boolean.TRUE.toString()); + } client().performRequest(createTemplateRequest); - if (isRunningAgainstOldCluster()) { // Create the repo XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject(); { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json index e3a97ee5c012a..0fa351b3812a7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json @@ -16,6 +16,10 @@ } }, "params": { + "include_type_name": { + "type" : "boolean", + "description" : "Whether a type should be returned in the body of the mappings. (default: true)" + }, "flat_settings": { "type": "boolean", "description": "Return settings in flat format (default: false)" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json index 5bcb2f8a24346..a56db2daf3ad2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json @@ -33,6 +33,10 @@ "flat_settings": { "type": "boolean", "description": "Return settings in flat format (default: false)" + }, + "include_type_name": { + "type" : "boolean", + "description" : "Whether a type is included in the body of the mappings. (default: true)" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml index b4e66c23c605b..e988a523c4e41 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml @@ -199,3 +199,79 @@ indices.put_template: name: test body: {} + +--- +"Put template with mappings and doc type": + - do: + indices.put_template: + name: test + body: + index_patterns: test-* + mappings: + _doc: + properties: + description: + type: "text" + - do: + indices.get_template: + name: test + - match: {test.index_patterns: ["test-*"]} + - match: {test.mappings._doc.properties.description.type: text} + +--- +"Put template with mappings and no doc type": + - skip: + version: " - 6.99.99" + reason: this uses a new API that has been added in 7.0 + - do: + indices.put_template: + name: test + include_type_name: false + body: + index_patterns: test-* + mappings: + properties: + description: + type: "text" + - do: + indices.get_template: + name: test + - match: {test.index_patterns: ["test-*"]} + - match: {test.mappings._doc.properties.description.type: text} + + # Also test we can GET without the type + - do: + indices.get_template: + name: test + include_type_name: false + - match: {test.index_patterns: ["test-*"]} + - match: {test.mappings.properties.description.type: text} + +--- +"Put template with mappings no doc type and no fields": + - skip: + version: " - 6.99.99" + reason: this uses a new API that has been added in 7.0 + - do: + indices.put_template: + name: test + include_type_name: false + body: + index_patterns: test-* + mappings: + _source: + enabled: false + - do: + indices.get_template: + name: test + - match: {test.index_patterns: ["test-*"]} + - match: {test.mappings._doc._source.enabled: false} + + # Also test we can GET without the type + - do: + indices.get_template: + name: test + include_type_name: false + - match: {test.index_patterns: ["test-*"]} + - match: {test.mappings._source.enabled: false} + diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java index cfaa9408da1e5..1ad9a8b98616d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesRequest.java @@ -34,6 +34,8 @@ public class GetIndexTemplatesRequest extends MasterNodeReadRequest { private String[] names; + + private boolean includeTypeNamesInResponse = true; public GetIndexTemplatesRequest() { } @@ -46,6 +48,14 @@ public GetIndexTemplatesRequest(StreamInput in) throws IOException { super(in); names = in.readStringArray(); } + + public boolean includeTypeNamesInResponse() { + return includeTypeNamesInResponse; + } + + public void includeTypeNamesInResponse(boolean includeTypeNamesInResponse) { + this.includeTypeNamesInResponse = includeTypeNamesInResponse; + } @Override public void writeTo(StreamOutput out) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index ce82d277dbbd7..0fc0a1d57b350 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -47,6 +47,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.io.UncheckedIOException; @@ -83,6 +84,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequestdoctype->properties JSON private Map mappings = new HashMap<>(); private final Set aliases = new HashSet<>(); @@ -209,15 +211,27 @@ public Settings settings() { } /** - * Adds mapping that will be added when the index gets created. + * Adds mapping that will be added when the index gets created using a custom doc type * * @param type The mapping type * @param source The mapping source * @param xContentType The type of content contained within the source + * @deprecated use {@link #mappingNoDocType(String, XContentType)} instead */ + @Deprecated public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) { - return mapping(type, new BytesArray(source), xContentType); + return internalMapping(type, XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2()); } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mappingNoDocType(String source, XContentType xContentType) { + return mapping(MapperService.SINGLE_MAPPING_NAME, new BytesArray(source), xContentType); + } /** * The cause for this index template creation. @@ -232,14 +246,27 @@ public String cause() { } /** - * Adds mapping that will be added when the index gets created. - * + * Adds mapping that will be added when the index gets created using a choice of + * custom document type + * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mappingNoDocType(XContentBuilder)} instead */ + @Deprecated public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { - return mapping(type, BytesReference.bytes(source), source.contentType()); + return internalMapping(type, XContentHelper.convertToMap(BytesReference.bytes(source), true, source.contentType()).v2()); } + + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + */ + public PutIndexTemplateRequest mappingNoDocType(XContentBuilder source) { + return mapping(MapperService.SINGLE_MAPPING_NAME, source); + } /** * Adds mapping that will be added when the index gets created. @@ -247,24 +274,47 @@ public PutIndexTemplateRequest mapping(String type, XContentBuilder source) { * @param type The mapping type * @param source The mapping source * @param xContentType the source content type + * @deprecated use {@link #mappingNoDocType(BytesReference, XContentType)} instead */ - public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - try { - mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType)); - return this; - } catch (IOException e) { - throw new UncheckedIOException("failed to convert source to json", e); - } + @Deprecated + public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) { + internalMapping(type, XContentHelper.convertToMap(source, true, xContentType).v2()); + return this; } - + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mappingNoDocType(BytesReference source, XContentType xContentType) { + return mapping(MapperService.SINGLE_MAPPING_NAME, source, xContentType); + } + /** * Adds mapping that will be added when the index gets created. * + * @param source The mapping source + */ + public PutIndexTemplateRequest mapping(Map source) { + return internalMapping(MapperService.SINGLE_MAPPING_NAME, source); + } + + /** + * Adds mapping that will be added when the index gets created using a custom + * document type name + * * @param type The mapping type * @param source The mapping source + * @deprecated use {@link #mapping(Map)} instead */ + @Deprecated public PutIndexTemplateRequest mapping(String type, Map source) { + return internalMapping(type, source); + } + + private PutIndexTemplateRequest internalMapping(String type, Map source) { // wrap it in a type map if its not if (source.size() != 1 || !source.containsKey(type)) { source = MapBuilder.newMapBuilder().put(type, source).map(); @@ -272,7 +322,13 @@ public PutIndexTemplateRequest mapping(String type, Map source) try { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.map(source); - return mapping(type, builder); + Objects.requireNonNull(builder.contentType()); + try { + mappings.put(type, XContentHelper.convertToJson(BytesReference.bytes(builder), false, false, builder.contentType())); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to json", e); + } } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } @@ -281,11 +337,22 @@ public PutIndexTemplateRequest mapping(String type, Map source) /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). + * @deprecated use {@link #simplifiedMapping(Object...)} instead */ + @Deprecated public PutIndexTemplateRequest mapping(String type, Object... source) { mapping(type, PutMappingRequest.buildFromSimplifiedDef(type, source)); return this; } + + /** + * A specialized simplified mapping source method, takes the form of simple properties definition: + * ("field1", "type=string,store=true"). + * Uses the default document type "_doc" + */ + public PutIndexTemplateRequest simplifiedMapping(Object... source) { + return mapping(MapperService.SINGLE_MAPPING_NAME, source); + } public Map mappings() { return this.mappings; @@ -293,7 +360,9 @@ public Map mappings() { /** * The template source definition. + * @deprecated use {@link #sourceNoDocTypes(XContentBuilder)} instead */ + @Deprecated public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { try { return source(BytesReference.bytes(templateBuilder), templateBuilder.contentType()); @@ -305,8 +374,35 @@ public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { /** * The template source definition. */ - @SuppressWarnings("unchecked") + public PutIndexTemplateRequest sourceNoDocTypes(XContentBuilder templateBuilder) { + try { + return sourceNoDocTypes(BytesReference.bytes(templateBuilder), templateBuilder.contentType()); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to build json for template request", e); + } + } + + /** + * The template source definition with no doc types expected in the mapping. + * @deprecated use {@link #sourceNoDocTypes(Map)} instead + */ + @Deprecated public PutIndexTemplateRequest source(Map templateSource) { + return source(templateSource, true); + } + + /** + * The template source definition with no doc types expected in the mapping. + */ + public PutIndexTemplateRequest sourceNoDocTypes(Map templateSource) { + return source(templateSource, false); + } + + /** + * The template source definition. + */ + @SuppressWarnings("unchecked") + private PutIndexTemplateRequest source(Map templateSource, boolean includeDocTypesInMapping) { Map source = templateSource; for (Map.Entry entry : source.entrySet()) { String name = entry.getKey(); @@ -339,13 +435,17 @@ public PutIndexTemplateRequest source(Map templateSource) { settings((Map) entry.getValue()); } else if (name.equals("mappings")) { Map mappings = (Map) entry.getValue(); - for (Map.Entry entry1 : mappings.entrySet()) { - if (!(entry1.getValue() instanceof Map)) { - throw new IllegalArgumentException( - "Malformed [mappings] section for type [" + entry1.getKey() + - "], should include an inner object describing the mapping"); + if (includeDocTypesInMapping == false) { + internalMapping(MapperService.SINGLE_MAPPING_NAME, mappings); + } else { + for (Map.Entry entry1 : mappings.entrySet()) { + if (!(entry1.getValue() instanceof Map)) { + throw new IllegalArgumentException( + "Malformed [mappings] section for type [" + entry1.getKey() + + "], should include an inner object describing the mapping"); + } + internalMapping(entry1.getKey(), (Map) entry1.getValue()); } - mapping(entry1.getKey(), (Map) entry1.getValue()); } } else if (name.equals("aliases")) { aliases((Map) entry.getValue()); @@ -355,34 +455,83 @@ public PutIndexTemplateRequest source(Map templateSource) { } return this; } - + /** - * The template source definition. + * Returns true if the mapping uses custom types. Is used in the HighLevelRestClient to manage + * formatting of requests to a server + */ + public boolean isCustomTyped () { + for (String key : mappings.keySet()) { + if(key.equals(MapperService.SINGLE_MAPPING_NAME) == false) { + return true; + } + } + return false; + } + + /** + * The template source definition with doc types in mapping + * @deprecated use {@link #sourceNoDocTypes(String, XContentType)} instead */ + @Deprecated public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { - return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true), true); + } + + /** + * The template source definition with no doc types expected in the mapping. + */ + public PutIndexTemplateRequest sourceNoDocTypes(String templateSource, XContentType xContentType) { + return sourceNoDocTypes(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); } /** - * The template source definition. + * The template source definition with doc types in mapping + * @deprecated use {@link #sourceNoDocTypes(byte[], XContentType)} instead */ + @Deprecated public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { return source(source, 0, source.length, xContentType); } + + /** + * The template source definition with no doc types expected in the mapping. + */ + public PutIndexTemplateRequest sourceNoDocTypes(byte[] source, XContentType xContentType) { + return sourceNoDocTypes(source, 0, source.length, xContentType); + } /** - * The template source definition. + * The template source definition with doc types in mapping + * @deprecated use {@link #sourceNoDocTypes(byte[], int, int, XContentType)} instead */ + @Deprecated public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { return source(new BytesArray(source, offset, length), xContentType); } + + /** + * The template source definition with no doc types expected in the mapping. + */ + public PutIndexTemplateRequest sourceNoDocTypes(byte[] source, int offset, int length, XContentType xContentType) { + return sourceNoDocTypes(new BytesArray(source, offset, length), xContentType); + } /** - * The template source definition. + * The template source definition with doc types in mapping + * @deprecated use {@link #sourceNoDocTypes(BytesReference, XContentType)} instead */ + @Deprecated public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { return source(XContentHelper.convertToMap(source, true, xContentType).v2()); } + + /** + * The template source definition with no doc types expected in the mapping. + */ + public PutIndexTemplateRequest sourceNoDocTypes(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2(), false); + } public Set aliases() { return this.aliases; @@ -529,15 +678,35 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws settings.toXContent(builder, params); builder.endObject(); - builder.startObject("mappings"); - for (Map.Entry entry : mappings.entrySet()) { - builder.field(entry.getKey()); - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, entry.getValue())) { - builder.copyCurrentStructure(parser); - } + if (isCustomTyped()) { + builder.startObject("mappings"); + for (Map.Entry entry : mappings.entrySet()) { + builder.field(entry.getKey()); + String mappingDef = entry.getValue(); + if(isCustomTyped() == false) { + // export as typeless mapping format - strip the _doc from the mappings->_doc->properties + String val = entry.getValue(); + Map mappingAsMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, val, true); + XContentBuilder mappingMinusTypeBuilder = XContentFactory.contentBuilder(XContentType.JSON); + mappingMinusTypeBuilder.map((Map) mappingAsMap.get(MapperService.SINGLE_MAPPING_NAME)); + mappingDef = BytesReference.bytes(mappingMinusTypeBuilder).utf8ToString(); + } + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingDef)) { + builder.copyCurrentStructure(parser); + } + } + builder.endObject(); + } else { + assert mappings.size() <= 1; // Highlander rule applies to doc types: there can be only one + for (Map.Entry entry : mappings.entrySet()) { + builder.field("mappings"); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, entry.getValue())) { + builder.copyCurrentStructure(parser); + } + } } - builder.endObject(); builder.startObject("aliases"); for (Alias alias : aliases) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index f9ef5786afdca..1071798e8ac12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -341,6 +341,8 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XCont public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { + // TODO make default include_type_name choice a constant on MapperService? See duplicate code in RestGetMappingAction + boolean includeTypeName = params.paramAsBoolean("include_type_name", true); builder.field("order", indexTemplateMetaData.order()); if (indexTemplateMetaData.version() != null) { @@ -361,8 +363,16 @@ public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData, // the type name is the root value, reduce it mapping = (Map) mapping.get(cursor.key); } - builder.field(cursor.key); - builder.map(mapping); + if (includeTypeName) { + builder.field(cursor.key); + builder.map(mapping); + } else { + // Pull all nested properties for doc type up a level + for (Map.Entry value : mapping.entrySet()) { + builder.field(value.getKey()); + builder.value(value.getValue()); + } + } } builder.endObject(); } else { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 38c1cb76611f4..12ec0239ab6e2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -58,8 +58,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final String[] names = Strings.splitStringByCommaToArray(request.param("name")); - + final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); + getIndexTemplatesRequest.includeTypeNamesInResponse(request.paramAsBoolean("include_type_name", true)); getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 258bb05a7d66c..1401b5637459f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -38,6 +38,9 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger( LogManager.getLogger(RestPutIndexTemplateAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying document types in template mappings is deprecated." + + "Use `include_type_name=false` parameter in REST requests to assert they are type-free."; + public RestPutIndexTemplateAction(Settings settings, RestController controller) { super(settings); @@ -63,7 +66,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout())); putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - putRequest.source(request.requiredContent(), request.getXContentType()); + if (request.paramAsBoolean("include_type_name", true)) { + putRequest.source(request.requiredContent(), request.getXContentType()); + // Check if a mapping was provided - sometimes a template can just be settings. + if (putRequest.isCustomTyped()) { + deprecationLogger.deprecated(TYPES_DEPRECATION_MESSAGE); + } + } else { + putRequest.sourceNoDocTypes(request.requiredContent(), request.getXContentType()); + } return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java index 779a737c88279..3e4a51771cdc5 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java @@ -455,6 +455,7 @@ public void testNonexistentPolicy() throws Exception { " }\n" + "}", ContentType.APPLICATION_JSON); Request templateRequest = new Request("PUT", "_template/test"); + templateRequest.addParameter("include_type_name", Boolean.FALSE.toString()); templateRequest.setEntity(template); client().performRequest(templateRequest);