Merge remote-tracking branch 'origin/master' into no-copy-settings-for-you

* origin/master:
  Update forcemerge.asciidoc (elastic#30113)
  Added zentity to the list of API extension plugins (elastic#29143)
  Fix the search request default operation behavior doc (elastic#29302) (elastic#29405)
  Watcher: Mark watcher as started only after loading watches (elastic#30403)
  Pass the task to broadcast actions (elastic#29672)
  Disable REST default settings testing until elastic#29229 is back-ported
  Correct wording in log message (elastic#30336)
  Do not fail snapshot when deleting a missing snapshotted file (elastic#30332)
  AwaitsFix testCreateShrinkIndexToN
  DOCS: Correct mapping tags in put-template api
  DOCS: Fix broken link in the put index template api
  Add put index template api to high level rest client (elastic#30400)
  Relax testAckedIndexing to allow document updating
  [Docs] Add snippets for POS stop tags default value
  Move respect accept header on no handler to 6.3.1

Author: jasontedor

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -51,6 +51,8 @@
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -456,4 +458,26 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Action
                 UpdateSettingsResponse::fromXContent, listener, emptySet(), headers);
     }
 
+    /**
+     * Puts an index template using the Index Templates API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     */
+    public PutIndexTemplateResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate,
+            PutIndexTemplateResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously puts an index template using the Index Templates API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API
+     * on elastic.co</a>
+     */
+    public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest,
+                                 ActionListener<PutIndexTemplateResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, RequestConverters::putTemplate,
+            PutIndexTemplateResponse::fromXContent, listener, emptySet(), headers);
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -47,6 +47,7 @@
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
@@ -77,7 +78,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
-import org.elasticsearch.rest.action.RestFieldCapabilitiesAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
@@ -86,10 +86,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Locale;
-import java.util.Map;
 import java.util.StringJoiner;
 
 final class RequestConverters {
@@ -647,6 +644,21 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr
         return request;
     }
 
+    static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_template").addPathPart(putIndexTemplateRequest.name()).build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout());
+        if (putIndexTemplateRequest.create()) {
+            params.putParam("create", Boolean.TRUE.toString());
+        }
+        if (Strings.hasText(putIndexTemplateRequest.cause())) {
+            params.putParam("cause", putIndexTemplateRequest.cause());
+        }
+        request.setEntity(createEntity(putIndexTemplateRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -56,11 +56,14 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -73,11 +76,19 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractRawValues;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
 import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -812,4 +823,59 @@ public void testIndexPutSettingNonExistent() throws IOException {
                 + "or check the breaking changes documentation for removed settings]"));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testPutTemplate() throws Exception {
+        PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest()
+            .name("my-template")
+            .patterns(Arrays.asList("pattern-1", "name-*"))
+            .order(10)
+            .create(randomBoolean())
+            .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0"))
+            .mapping("doc", "host_name", "type=keyword", "description", "type=text")
+            .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz"));
+
+        PutIndexTemplateResponse putTemplateResponse = execute(putTemplateRequest,
+            highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync);
+        assertThat(putTemplateResponse.isAcknowledged(), equalTo(true));
+
+        Map<String, Object> templates = getAsMap("/_template/my-template");
+        assertThat(templates.keySet(), hasSize(1));
+        assertThat(extractValue("my-template.order", templates), equalTo(10));
+        assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*"));
+        assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3"));
+        assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0"));
+        assertThat(extractValue("my-template.mappings.doc.properties.host_name.type", templates), equalTo("keyword"));
+        assertThat(extractValue("my-template.mappings.doc.properties.description.type", templates), equalTo("text"));
+        assertThat((Map<String, String>) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc"));
+        assertThat((Map<String, String>) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz"));
+    }
+
+    public void testPutTemplateBadRequests() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        // Failed to validate because index patterns are missing
+        PutIndexTemplateRequest withoutPattern = new PutIndexTemplateRequest("t1");
+        ValidationException withoutPatternError = expectThrows(ValidationException.class,
+            () -> execute(withoutPattern, client.indices()::putTemplate, client.indices()::putTemplateAsync));
+        assertThat(withoutPatternError.validationErrors(), contains("index patterns are missing"));
+
+        // Create-only specified but an template exists already
+        PutIndexTemplateRequest goodTemplate = new PutIndexTemplateRequest("t2").patterns(Arrays.asList("qa-*", "prod-*"));
+        assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged());
+        goodTemplate.create(true);
+        ElasticsearchException alreadyExistsError = expectThrows(ElasticsearchException.class,
+            () -> execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync));
+        assertThat(alreadyExistsError.getDetailedMessage(),
+            containsString("[type=illegal_argument_exception, reason=index_template [t2] already exists]"));
+        goodTemplate.create(false);
+        assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged());
+
+        // Rejected due to unknown settings
+        PutIndexTemplateRequest unknownSettingTemplate = new PutIndexTemplateRequest("t3")
+            .patterns(Collections.singletonList("any"))
+            .settings(Settings.builder().put("this-setting-does-not-exist", 100));
+        ElasticsearchStatusException unknownSettingError = expectThrows(ElasticsearchStatusException.class,
+            () -> execute(unknownSettingTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync));
+        assertThat(unknownSettingError.getDetailedMessage(), containsString("unknown setting [index.this-setting-does-not-exist]"));
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -26,12 +26,11 @@
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
@@ -46,10 +45,11 @@
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -70,22 +70,21 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestConverters.EndpointBuilder;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.client.RequestConverters.EndpointBuilder;
-import org.elasticsearch.client.RequestConverters.Params;
 import org.elasticsearch.index.RandomCreateIndexGenerator;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -94,7 +93,6 @@
 import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
-import org.elasticsearch.rest.action.RestFieldCapabilitiesAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
@@ -111,8 +109,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -121,7 +117,6 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 import java.util.StringJoiner;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -1432,6 +1427,48 @@ public void testIndexPutSettings() throws IOException {
         assertEquals(expectedParams, request.getParameters());
     }
 
+    public void testPutTemplateRequest() throws Exception {
+        Map<String, String> names = new HashMap<>();
+        names.put("log", "log");
+        names.put("template#1", "template%231");
+        names.put("-#template", "-%23template");
+        names.put("foo^bar", "foo%5Ebar");
+
+        PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest()
+            .name(randomFrom(names.keySet()))
+            .patterns(Arrays.asList(generateRandomStringArray(20, 100, false, false)));
+        if (randomBoolean()) {
+            putTemplateRequest.order(randomInt());
+        }
+        if (randomBoolean()) {
+            putTemplateRequest.version(randomInt());
+        }
+        if (randomBoolean()) {
+            putTemplateRequest.settings(Settings.builder().put("setting-" + randomInt(), randomTimeValue()));
+        }
+        if (randomBoolean()) {
+            putTemplateRequest.mapping("doc-" + randomInt(), "field-" + randomInt(), "type=" + randomFrom("text", "keyword"));
+        }
+        if (randomBoolean()) {
+            putTemplateRequest.alias(new Alias("alias-" + randomInt()));
+        }
+        Map<String, String> expectedParams = new HashMap<>();
+        if (randomBoolean()) {
+            expectedParams.put("create", Boolean.TRUE.toString());
+            putTemplateRequest.create(true);
+        }
+        if (randomBoolean()) {
+            String cause = randomUnicodeOfCodepointLengthBetween(1, 50);
+            putTemplateRequest.cause(cause);
+            expectedParams.put("cause", cause);
+        }
+        setRandomMasterTimeout(putTemplateRequest, expectedParams);
+        Request request = RequestConverters.putTemplate(putTemplateRequest);
+        assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name())));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+        assertToXContentBody(putTemplateRequest, request.getEntity());
+    }
+
     private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
         BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, REQUEST_BODY_CONTENT_TYPE, false);
         assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue());